{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0. 环境设定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import sys\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize(shape, stddev=0.1):\n",
    "    return tf.truncated_normal(shape, stddev=stddev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '../data/mnist/input_data/'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist.train.images.shape,mnist.train.labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist.test.images.shape,mnist.test.labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 准备placeholder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=tf.placeholder(tf.float32,[None,784],name='X_placeholder')\n",
    "Y=tf.placeholder(tf.float32,[None,10],name='Y_placeholder')\n",
    "init_learning_rate = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 准备参数/权重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_hidden1=200\n",
    "n_hidden2=10\n",
    "n_input=784\n",
    "n_classes=10\n",
    "\n",
    "weights={\n",
    "    'W1':tf.Variable(initialize([n_input,n_hidden1],stddev=np.sqrt(2/784)),name='W1'),\n",
    "    'W2':tf.Variable(initialize([n_hidden1,n_hidden2],stddev=np.sqrt(2/784)),name='W2'),\n",
    "}\n",
    "biases={\n",
    "    'b1':tf.Variable(tf.random_normal([n_hidden1]),name='b1'),\n",
    "    'b2':tf.Variable(tf.random_normal([n_hidden2]),name='b2'),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. 构建计算graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mutilayer_perception(x,weights,biases):\n",
    "    logits_1=tf.add(tf.matmul(x,weights['W1']),biases['b1'])\n",
    "    output_1=tf.nn.relu(logits_1)\n",
    "    logits_2=tf.add(tf.matmul(output_1,weights['W2']),biases['b2'])\n",
    "    out_layer=logits_2\n",
    "    return out_layer\n",
    "\n",
    "pred=mutilayer_perception(X,weights,biases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. 计算loss和accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y,logits=pred))\n",
    "l2_loss=tf.nn.l2_loss(weights['W1'])+tf.nn.l2_loss(weights['W2'])\n",
    "total_loss=cross_entropy+4e-5*l2_loss\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. 设置optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#设置learning_rate迭代方式\n",
    "epoch_steps = tf.to_int64(tf.div(65000, tf.shape(X)[0]))\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "current_epoch = global_step//epoch_steps\n",
    "decay_times = current_epoch \n",
    "current_learning_rate = tf.multiply(init_learning_rate, \n",
    "                                    tf.pow(0.65, tf.to_float(decay_times)))\n",
    "#设置optimizer\n",
    "optimizer = tf.train.AdamOptimizer(current_learning_rate)\n",
    "gradients = optimizer.compute_gradients(total_loss)\n",
    "train_step = optimizer.apply_gradients(gradients)\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(\n",
    "    current_learning_rate).minimize(\n",
    "    total_loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. 初始化变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "init=tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. 在session中执行graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for step in range(5000):\n",
    "        batch_x,batch_y= mnist.train.next_batch(100)\n",
    "        lr = 1e-2\n",
    "        _, loss, l2_loss_value, total_loss_value, current_lr_value = \\\n",
    "        sess.run([train_step, cross_entropy, l2_loss, total_loss, current_learning_rate],\n",
    "               feed_dict={X: batch_x, Y: batch_y, init_learning_rate:lr})\n",
    "  \n",
    "        if (step+1) % 100 == 0:\n",
    "            print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f, current_lr_value:%f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value,current_lr_value))\n",
    "    #print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "            print(sess.run(accuracy, feed_dict={X: mnist.test.images, Y: mnist.test.labels}))\n",
    "    #print(current_lr_value)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
